#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 20-6-29 下午7:25
# @Author  : 邹顺利
# @Email   : 409358374@qq.com
# @File    : threadCrawler.py
# @Software: PyCharm

import threading

from queue import Queue

from lxml import etree

import requests
import random
import json

CRAWL_EXIT = False
PARSE_EXIT = False


class ThreadCrawl(threading.Thread):
    def __init__(self, threadName, pageQueue, dataQueue):
        super(ThreadCrawl, self).__init__()
        self.threadName = threadName
        self.pageQueue = pageQueue
        self.dataQueue = dataQueue
        self.user_agent = [
            'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0'
        ]
        self.headers = {'User-Agent': random.choice(self.user_agent)}

    def run(self):
        print('Start', self.threadName)
        while not CRAWL_EXIT:
            try:
                # 取出一个数字， 先进先出
                # 可选参数block，默认值为True
                # 如果队列为空, block为True,不会结束,会进入阻塞状态, 直到队列有新的数据
                # 如果队列为空, block为False,会弹出一个Queue.empty()异常
                page = self.pageQueue.get(False)
                url = "http://www.qiushibaike.com/8hr/page/" + str(page) + "/"
                r = requests.get(url, headers=self.headers, timeout=2)
                content = r.content
                self.dataQueue.put(content)
            except Exception as e:
                print('Exception-->', e)

        print('End', self.threadName)


class ThreadParse(threading.Thread):
    def __init__(self, threadName, dataQueue, filename, lock):
        super(ThreadParse, self).__init__()
        self.threadName = threadName
        self.dataQueue = dataQueue
        self.filename = filename
        self.lock = lock

    def run(self):
        print('Start', self.threadName)
        while not PARSE_EXIT:
            try:
                html = self.dataQueue.get(False)
                self.parseData(html)
            except Exception as e:
                print('Exception-->', e)

        print('End', self.threadName)

    def parseData(self, html):
        html = etree.HTML(html)
        node_list = html.xpath('//div[contains(@id, "qiushi_tag")]')

        for node in node_list:
            # xpath返回的列表，这个列表就这一个参数，用索引方式取出来，用户名
            username = node.xpath('./div/a/@title')[0]
            # 图片连接
            image = node.xpath('.//div[@class="thumb"]//@src')  # [0]
            # 取出标签下的内容,段子内容
            content = node.xpath('.//div[@class="content"]/span')[0].text
            # 取出标签里包含的内容，点赞
            zan = node.xpath('.//i')[0].text
            # 评论
            comments = node.xpath('.//i')[1].text

            items = {
                "username": username,
                "image": image,
                "content": content,
                "zan": zan,
                "comments": comments
            }

            with self.lock:
                # 写入存储的解析后的数据
                data = json.dumps(items, ensure_ascii=False).encode('utf-8')
                self.filename.write(data + '\n')


def main():
    # 页码的队列，表示十个页面
    pageQueue = Queue(10)
    # 放入1-10的数字， 先进先出
    for i in (1, 11):
        pageQueue.put(i)
    # Queue()参数为空是表示不限制 采集结果（每页的HTML源码）的数据队列
    dataQueue = Queue()

    # 三个采集线程的名字
    crawlList = ["crawlerOne", 'crawlerTwo', 'crawlerThree']

    # 存储三个采集线程的列表集合
    threadCrawl = list()
    for threadName in crawlList:
        thread = ThreadCrawl(threadName, pageQueue, dataQueue)
        thread.start()
        threadCrawl.append(thread)


    # 三个解析线程的名字
    parselList = ["parseOne", 'parseTwo', 'parseThree']

    # 存储三个解析线程的列表集合
    threadParse = list()
    for threadName in parselList:
        thread = ThreadCrawl(threadName, pageQueue, dataQueue)
        thread.start()
        threadParse.append(thread)

    # 等待pageQueue队列为空，也就是等待之前的操作执行完毕
    while not pageQueue.empty():
        pass
    global CRAWL_EXIT
    CRAWL_EXIT = True
    print("pageQueue为空")

    for thread in threadCrawl:
        thread.join()
        print('1')



    # 等待dataQueue队列为空，也就是等待之前的操作执行完毕
    while not dataQueue.empty():
        pass
    global PARSE_EXIT_EXIT
    CRAWL_EXIT = True
    print("pageQueue为空")

    for thread in threadCrawl:
        thread.join()
        print('1')


if __name__ == '__main__':
    main()
